** Prepare file with decomposition of variety-adjusted inflation difference by quality modules

** Compute Differential Inflation Across Quality-Modules

clear
set obs 1 
generate ces_price_index_final=.
save "$db/Important Price Datasets/feenstra_ces_inflation_income_acrossQM.dta", replace

foreach i of numlist 2004(1)2014 {

display "Starting Tornqvist inflation difference for year `i', QM"

* first, compute spending shares in each module by quality deciles 
local var=`i'+1

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear
rename total_quantity total_quantity_old
rename average_unit_price average_unit_price_old
rename average_unit_price_all average_unit_price_all_old

merge 1:1 upc upc_ver income_quintile using "$db/Important Price Datasets/price_index_household_income_`var'_2004_modules_final.dta"
keep if _merge==3
drop total_spending
drop _merge

* bring in info on UPC price decile (within product module) 
merge m:1 upc upc_ver_uc using "$db/Important Lists/upc_brand_size_final.dta"
keep if _merge==3
drop _merge
sum size1_amount, d
drop if size1_amount<r(p1) | size1_amount>r(p99)
gen average_unit_price_adj=(average_unit_price_all+average_unit_price_all_old)/(2*size1_amount*multi)
* create quality ranks without weights
sort product_module_code average_unit_price_adj
bysort product_module_code: gen double temp=[_N]
bysort product_module_code: gen double temp2=[_n]
gen rank=temp2/temp*100
gen quality_rank=.
foreach r of numlist 1(1)10 {
replace quality_rank=`r' if rank<=`r'*10 & missing(quality_rank)
}

gen double price_ratio=average_unit_price/average_unit_price_old
gen double price_ratio_all=average_unit_price_all/average_unit_price_all_old
* get rid of outliers
sum price_ratio, d
drop if price_ratio<r(p1) | price_ratio>r(p99)
*sum price_ratio_all, d
*drop if price_ratio_all<r(p1) | price_ratio_all>r(p99)

* compute spending shares by product module - quality
bysort product_module quality income_quintile: egen double total_spending_inc_QM=sum(average_unit_price*total_quantity)
bysort product_module quality income_quintile: egen double total_spending_inc_QM_old=sum(average_unit_price_old*total_quantity_old)

keep product_module quality income_quintile total_spending_inc_QM total_spending_inc_QM_old
duplicates drop

* second, bring to file with inflation by quality-modules
merge m:1 product_module quality using "$db/Important Price Datasets/inflation_QM_`i'_2yrs.dta"
keep if _merge==3
drop _merge

* adjust CES inflation with new goods
merge m:1 product_module quality using "$db/Important Price Datasets/new_exit_QM_`var'.dta"
keep if _merge==3
drop _merge

gen feenstra=(1-ssnp)/(1-ssep)
* there is a small number of obs for which ssnp=1 or ssep=0, then we can't compute feenstra
replace ssnp=0 if missing(feenstra)
replace ssep=0 if missing(feenstra)
replace feenstra=1 if missing(feenstra)
* winsorize feenstra at 5% level (by department/year)
bysort department_code year: egen p5 = pctile(feenstra), p(5)
bysort department_code year: egen p95 = pctile(feenstra), p(95)
gen feenstra_win = feenstra
replace feenstra_win = p95 if feenstra>p95
replace feenstra_win = p5 if feenstra<p5

merge m:1 product_module_code using "$db/Important Price Datasets/sigmas", keepusing(sigma_common)
drop if _merge==2
drop _merge

foreach i in feenstra_win {
gen `i'_common=`i'^(1/(sigma_common-1))
}

* create full CES index 
gen full_ces_price_index = ces_price_index*feenstra_win_common

* now compute CES inflation with variety adjustment
bysort income_quintile: egen double total_spending_inc=sum(total_spending_inc_QM)
bysort income_quintile: egen double total_spending_inc_old=sum(total_spending_inc_QM_old)
gen double share_inc_QM=total_spending_inc_QM/total_spending_inc
gen double share_inc_QM_old=total_spending_inc_QM_old/total_spending_inc_old
gen weight_num_QM=(share_inc_QM-share_inc_QM_old)/(ln(share_inc_QM)-ln(share_inc_QM_old))
bysort income_quintile: egen double weight_den_QM=sum(weight_num_QM)
gen double weight_QM=weight_num_QM/weight_den_QM
gen double numerator_temp=weight_QM*log(full_ces_price_index)
bysort income_quintile: egen double numerator=sum(numerator_temp)
gen double full_ces_price_index_final=exp(numerator)

* ces without variety adjustment
gen double numerator_temp_v2=weight_QM*log(ces_price_index)
bysort income_quintile: egen double numerator_v2=sum(numerator_temp_v2)
gen double ces_price_index_final=exp(numerator_v2)

keep ces_price_index_final full_ces_price_index_final income_quintile
duplicates drop
gen year=`var'

append using "$db/Important Price Datasets/feenstra_ces_inflation_income_acrossQM.dta"
drop if missing(ces_price_index_final)
save "$db/Important Price Datasets/feenstra_ces_inflation_income_acrossQM.dta", replace 
}
